#!/usr/bin/env python
#coding: utf-8

"""
主程序，把数据都写入队列中
"""

from Download import request
from mongodb_queue import MogoQueue
from bs4 import BeautifulSoup


spider_queue = MogoQueue('meinvxiezhenji', 'crawl_queue')
def start(url):
    """把url写入队列中"""
    response = request.get(url, 3)
    Soup = BeautifulSoup(response.text, 'lxml')
    # 查找到 div class='all'中所有的a标签(通过观察原网页可以发现，a标签中只包含个人写真集)，并存储到队列中
    all_a = Soup.find('div', class_='all').find_all('a')
    for a in all_a:
        title = a.get_text()
        url = a['href']
        spider_queue.push(url, title)

if __name__ == "__main__":
    start('http://www.mzitu.com/all')